import scrapy

from MoocSpider.MoocCourseItems import MoocCourseItem


class MySpider(scrapy.Spider):
    name = "Mooc"
    allowed_domains = ["imooc.com"]
    start_urls = ["http://www.imooc.com/course/list"]
    def parse(self, response):
        item = MoocCourseItem()  # 实例容器保存爬取信息
        # 爬取数据
        for box in response.xpath('//div[@class="course-card-container"]/a[@target="_blank"]'):
            item['url'] = 'http://www.imooc.com' + box.xpath('.//@href').extract()[0]
            item['image_url'] = box.xpath('.//div[@class="course-card-top"]/img/@src').extract()[0]
            for itemnode in box.xpath('.//div[@class="course-card-content"]'):
                item['title'] = itemnode.xpath('.//h3/text()').extract()[0].strip()
                item['class_level'] = itemnode.xpath('.//span/text()').extract()[0].strip()
                item['student'] = itemnode.xpath('.//span/text()').extract()[1].strip()
                item['introduction'] = itemnode.xpath('.//p/text()').extract()[0].strip()
           # 返回爬取的数据
            yield item
        # 获取下一页信息
        url = response.xpath("//a[contains(text(),'下一页')]/@href").extract()
        if url:
            page = 'http://www.imooc.com' + url[0]
            yield scrapy.Request(page, callback=self.parse)
